#Read and mean impute the training data
setwd("~/accept_decline")

mean_impute = function(x)
{
	mu = mean(na.omit(x))
	x[is.na(x)] = mu
	x
}

df = readRDS("train.rds")
df = data.frame(lapply(df, mean_impute))
df = data.frame(df, loss_ind=df$loss>0)

#Compute the Tantrev variable
m = glm(formula = loss_ind ~ f274 + f528, family = binomial(), data = df)
pred_tantrev = predict(m, type="response")
a = cut(pred_tantrev, quantile(pred_tantrev, seq(0, 1, .01), include.lowest=T))
plot(tapply(df$loss_ind, a, mean))
sum(df$loss_ind[pred_tantrev>.09])/sum(df$loss_ind)

quantilize = function(x)
{
  qntls = unique(quantile(x, seq(0, 1, .05)))
  print(length(qntls))
  if(length(qntls)>1)cut(x, qntls, 1:(length(qntls) - 1), include.lowest=T)
  else as.factor(rep(1, length(x)))
}

df2 = df[pred_tantrev > .09,]
pred_tantrev2 = pred_tantrev[pred_tantrev > .09]
a = cut(pred_tantrev2, quantile(pred_tantrev2, seq(0, 1, .2), include.lowest=T))
df_quantile = lapply(df2[,2:(ncol(df2) - 2)], quantilize)
df_quantile = data.frame(df_quantile, tantrev = a, loss_ind=df2$loss_ind, loss=df2$loss)

#What if I just do quantile regression on the loss here?
#m_fine = glm(loss_ind ~ tantrev + f13 + f67 + f84 + f404 + f514 + f527 + f201 + f211 + f238,
#             data=df_quantile, family=binomial)

m_fine = glm(loss_ind~ tantrev+f2+f13+
			     f404+f468+f471+f543+f588+f597+
			     f611+f615+f670+f674+f681+f683+
				 f712+f715+f726+f727+f772,
	data=df_quantile, family=binomial)
pred = predict(m_fine, type="response", newdata=df_quantile)
pred[is.na(pred)] = 0
plot(quantile(pred, seq(0, 1, .05)), type="l")
b=cut(pred, quantile(pred, seq(0, 1, .05), include.lowest=T))
points(tapply(df_quantile$loss_ind, b, mean))
tapply(df_quantile$loss, b, median)


##Now need to apply to test, which means test variables must be quantilized
quantilize2 = function(x,y)
{
	qntls = unique(quantile(x, seq(0, 1, .05)))
	print(length(qntls))
	if(length(qntls)>1)cut(y, qntls, 1:(length(qntls) - 1), include.lowest=T)
	else as.factor(rep(1, length(y)))
}

df_test = readRDS("test.rds")


tantrev+f2+f13+
			     f404+f468+f471+f543+f588+f597+
			     f611+f615+f670+f674+f681+f683+
				 f712+f715+f726+f727+f772,

df_test = data.frame(id=df_test$id, lapply(
	df_test[ ,c("f2","f13",
			     "f404","f468","f471","f543","f588","f597",
			     "f611","f615","f670","f674","f681","f683",
				 "f712","f715","f726","f727","f772", "f274", "f528")],	mean_impute))
pred_test_tantrev = predict(m, newdata=df_test, type="response")
df_test2 = df_test[pred_test_tantrev>.09,]
pred_test_tantrev2 = pred_test_tantrev[pred_test_tantrev>.09]
b = cut(pred_test_tantrev2, quantile(pred_tantrev2, seq(0, 1, .2), include.lowest=T))

df_test_quantile = data.frame(id=df_test2$id, 
f2=quantilize2(df2$f2, df_test2$f2),
f13=quantilize2(df2$f13, df_test2$f13),
f404=quantilize2(df2$f404, df_test2$f404),
f468=quantilize2(df2$f468, df_test2$f468),
f471=quantilize2(df2$f471, df_test2$f471),
f543=quantilize2(df2$f543, df_test2$f543),
f588=quantilize2(df2$f588, df_test2$f588),
f597=quantilize2(df2$f597, df_test2$f597),
f611=quantilize2(df2$f611, df_test2$f611),
f615=quantilize2(df2$f615, df_test2$f615),
f670=quantilize2(df2$f670, df_test2$f670),
f674=quantilize2(df2$f674, df_test2$f674),
f681=quantilize2(df2$f681, df_test2$f681),
f683=quantilize2(df2$f683, df_test2$f683),
f712=quantilize2(df2$f712, df_test2$f712),
f715=quantilize2(df2$f715, df_test2$f715),
f726=quantilize2(df2$f726, df_test2$f726),
f727=quantilize2(df2$f727, df_test2$f727),
f772=quantilize2(df2$f772, df_test2$f772),
tantrev = b)

pred_test = predict(m_fine, type="response", newdata=df_test_quantile)

test_loss = rep(0, nrow(df_test2))

      (0,0.00233] (0.00233,0.00951]  (0.00951,0.0245]   (0.0245,0.0484] 
                0                 0                 0                 0 
  (0.0484,0.0951]    (0.0951,0.198]     (0.198,0.325]     (0.325,0.482] 
                0                 0                 0                 0 
     (0.482,0.63]      (0.63,0.742]     (0.742,0.819]     (0.819,0.867] 
                2                 3                 3                 4 
    (0.867,0.902]     (0.902,0.925]     (0.925,0.943]     (0.943,0.957] 
                4                 4                 5                 5 
    (0.957,0.968]     (0.968,0.978]     (0.978,0.988]         (0.988,1] 
                5                 5                 6                 6 


test_loss[pred_test>.482 & pred_test<=.63] = 2
test_loss[pred_test>.63 & pred_test<=.819] =3
test_loss[pred_test>.819 & pred_test<=.925] = 4
test_loss[pred_test>.925 & pred_test<=.978] = 5
test_loss[pred_test>.978] = 6



df_ans = data.frame(id=df_test2$id, test_loss)
df_ans2 = merge(data.frame(id=df_test$id), df_ans, by="id", all.x=T)
names(df_ans2)=c("id","loss")
df_ans2$loss[is.na(df_ans2$loss)] = 0

write.csv(df_ans2,"submission.csv", row.names=F, quote=F)
